# -*- coding: utf-8 -*-
import time
from scrapy_redis.spiders import RedisSpider
from bs4 import BeautifulSoup
from ..items import NewssiteItem


class SinaNewsSpider(RedisSpider):
    name = 'sinaNews'
    allowed_domains = ['sina.com.cn']
    custom_settings = {
        'ITEM_PIPELINES': {
            'news_spiders.pipelines.NewssitePipeline': 200,
        }
    }

    def parse(self, response):
        soup = BeautifulSoup(response.body, 'lxml')
        item = NewssiteItem()
        title = soup.select_one('.main-title')
        if title is None:
            print('[网页格式错误]')
            return

        item['title'] = title.text

        item['url'] = response.url

        # 获取正文内容
        content = soup.select('#article > p')
        item['content'] = ''
        for line in content:
            line = line.text.strip().replace(u'\u3000', ' ').replace(u'\xa0', ' ')
            if line.startswith('原标题'):
                continue
            item['content'] += line
        # print(item['content'])
        try:
            # 原标题
            firstLine = content[0].text.lstrip().replace(
                u'\u3000', '').replace(u'\xa0', '')
            if firstLine.startswith('原标题：'):
                item['sourceTitle'] = firstLine[4:]
        except Exception:
            item['content'] = soup.select_one(
                '#article').get_text("", strip=True)
        date = soup.select_one('.date-source > span')
        if date is not None:
            item['date'] = time.strftime(
                '%Y-%m-%d %H:%M:%S', time.strptime(date.text, r'%Y年%m月%d日 %H:%M'))

        source = soup.select_one('.source')
        if source is not None:
            item['source'] = source.text
        yield item
        return
